# -*- coding: utf-8 -*-
'''
Created on Sep 8, 2013

@author: LONG HOANG GIANG
'''

from urlparse import urljoin
import logging
import os
import re
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
import lib
import json
import workerpool

logging.basicConfig(level=logging.DEBUG, format='%(levelname)s :: %(asctime)s :: %(message)s', datefmt='%d/%m/%Y %H:%M:%S')

class Thread(workerpool.Job):
    
    def __init__(self, func, chapter, path):
        self.func = func
        self.chapter = chapter
        self.path = path
    
    def run(self):
        self.func(self.chapter, self.path)

class Hixx():
    
    
    def getDetail(self, item, path):
        url = item['url']
        tree = lib.Web.load(url).build_tree()
        contentNode = tree.xpath("//div[@id='content']/table[2]")[0]
        lib.Etree.cleanNode("./tr[1]", contentNode)
        text = lib.html2text(contentNode, textmode=False)
        textlen = len(text)
        text = '''<h4 class='title'>{0}</h4><div id="content">{1}</div>'''.format(item['name'], text)
        if textlen < 150:
            html = lib.Etree.tostring(contentNode)
            if re.search(r"arr_image\[1\]", html):
                images = lib.findIter(r"arr_image\[\d+\] = \"([^\"]+)\";", html, 1)
                if len(images) == 0:
                    raise
                text = '''<h4 class='title'>{0}</h4><div id="content">'''.format(item['name'])
                for img in images:
                    text += '''<p><img src="{0}" /></p>'''.format(img)
                text += "</div>"
        
        print text[:200]    
        lib.gz_file_put_content(json.dumps(text), path + "/files/" + item['path'])
        return text
    
    def getListChapter(self, url):
        chapters = []
        page = 1
        while page <= 1000:
            tree = lib.Web.load(url).build_tree()
            navigationNode = tree.xpath("//div[@class='bt_pagination']")
            n = len(navigationNode)
            if n == 0:
                xpathStr = "//div[@class='danh_sach']/a"
            else:
                xpathStr = "//div[@class='bt_pagination'][1]/following-sibling::div[@class='danh_sach']/a"
            for node in tree.xpath(xpathStr):
                name = lib.stringify(node)
                name = re.sub(r"--\d+--", "", name)
                name = re.sub(r"[.]", "", name)
                name = re.sub(r"(\s*)(\:)(?=[^ ])", ": ", name)
                name = lib.toUpper(name)
                link = node.get('href', '')
                if link == '': continue
                link = urljoin(url, link)
                chapters.append({'name': name, 'url': link, 'path': lib.crc32unsigned(link)})
            if len(navigationNode) == 0: break
            node = tree.xpath("//div[@class='bt_pagination']/div[@class='active']/following-sibling::*/a")
            if len(node) == 0: break
            pNext = node[0].get('href')
            if pNext == '': break
            url = urljoin(url, pNext)
            page += 1
        return chapters

    def crawlOneStory(self, url):
        chapters = self.getListChapter(url)
        folder = "/longhoanggiang/truyentext/{0}".format(lib.extractText(r"/([^/]+)\.html", url, 1)).lower()
        lib.makedir(folder + '/' + 'files')

        pool = workerpool.WorkerPool(size=2)
          
        for chapter in chapters:
            pool.put(Thread(self.getDetail, chapter, folder))
        pool.shutdown()
        pool.wait()
              
        for chapter in chapters:
            del chapter['url']
        lib.gz_file_put_content(lib.encryptCipher(json.dumps(chapters)), folder + "/data")

if __name__ == '__main__':
    
    
#     hixx = Hixx()
#     hixx.getDetail({'name': '', 'url': 'http://truyen.hixx.info/truyen/truyen-kiem-hiep/263986/Sat-Than/Chuong-1443-Khong-Gian-Thap-Tu-Tram-.html'}, '')
    
    url = raw_input("Enter URL: ")
    if url.startswith("http"):
     
        hixx = Hixx()
        hixx.crawlOneStory(url)
     
    logging.info("finished")
    os._exit(1)
    
    
    
    
